# -*- coding: utf-8 -*-
# @Time    : 2019/3/20 11:06
# @Author  : Macher
# @File    : getemail.py
# @Software: PyCharm Community Edition
import requests
from bs4 import BeautifulSoup
import mysql.connector

config = {'host': '219.216.65.41',
          'user': 'root',
          'password': '123456',
          'port': 3306,
          'database': 'mydata',
          'charset': 'utf8'}
class getemail:

    def __init__(self, url,sql):
        self.url = url
        self.sql = sql

    def insertMysql(self,dict1):
        conn = mysql.connector.connect(**config)
        cursor = conn.cursor()
        cursor.execute(self.sql, dict1)
        conn.commit()
        cursor.close()
        conn.close()

    def get_URL(self):
        headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 '
                                     '(KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'}
        session = requests.Session()
        session.trust_env = False
        r = session.get("%s" % (self.url),headers=headers)
        # r = requests.get("%s" % (url),headers=headers)
        # print r.text.encode('GBK','ignore').decode('GBk')
        return r.text

    def deal_email(self,pagesource):
        soup = BeautifulSoup(pagesource, 'lxml')
        tds = soup.find_all(class_ = 'blob-code blob-code-inner js-file-line')
        # print tds
        return tds

if __name__ == '__main__':
    url1 = 'https://github.com/martenson/disposable-email-domains/blob/master/disposable_email_blocklist.conf'
    url2 = 'https://raw.githubusercontent.com/WSTNPHX/scripts-n-tools/master/malware-email-addresses.txt'
    sql = 'insert into malware_email(email,source)values( %(email)s,%(source)s)'
    email1 = getemail(url1,sql)
    pagesource = email1.get_URL()
    items = email1.deal_email(pagesource)
    d1 = {}
    for item in items:
        d1['email'] = item.text
        d1['source'] = 'disposable_email_blocklist'
        email1.insertMysql(d1)
        print d1
    email2 = getemail(url2,sql)
    emails = email2.get_URL()
    for email in emails.split('\n'):
        d1['email'] = email
        d1['source'] = 'githubusercontent'
        email2.insertMysql(d1)
        print d1
